package com.imooc.scala.log

import org.apache.spark.sql.{SaveMode, SparkSession}

object SparkStatCleanJobJar {

  def main(args: Array[String]): Unit = {
    if (args.length != 2) {
      println("Usage: SparkStatCleanJobJar <inputPath> <outputPath> ")
      System.exit(1)
    }
    val Array(inputPath, outputPath) = args
    //第2步
    val spark = SparkSession.builder().getOrCreate()

    val accessRDD = spark.sparkContext.textFile(inputPath)
    //    accessRDD.take(10).foreach(println)
    val accessDF = spark.createDataFrame(accessRDD.map(x => AccessConvertUtil.parseLog(x)), AccessConvertUtil.struct)
    //    accessDF.printSchema()
    //    accessDF.show(false)
    accessDF.coalesce(1).write.format("parquet").mode(SaveMode.Overwrite).partitionBy("day").save(outputPath)
    spark.stop()

  }
}
